using System;
using System.Collections.Generic;
using System.Text;
using System.IO;
using Lucene.Net.Analysis.Standard;
using Lucene.Net.Analysis;

namespace FutureOpen.Core.func.analyzer
{
    using FutureOpen.Core.func.util;
    /// <summary>
    /// 
    /// </summary>
    public class ShootAnalyzer:Analyzer
    {
        //private System.Collections.Hashtable stopSet;
        public static System.String[] CHINESE_ENGLISH_STOP_WORDS  = null;
        public string NoisePath = ConfigUtil.GetStopwordPath();

        public ShootAnalyzer()
        {
            if (CHINESE_ENGLISH_STOP_WORDS == null)
            {
               
                List<string> list = new List<string>();
                                
                StreamReader reader = new StreamReader(NoisePath, System.Text.Encoding.UTF8);
                string noise = reader.ReadLine().Trim();
                while (!string.IsNullOrEmpty(noise))
                {
                    list.Add(noise.Trim());
                    noise = reader.ReadLine();
                }

                 CHINESE_ENGLISH_STOP_WORDS = list.ToArray();
            }
           
        }
      
        /// <summary>Constructs a {@link StandardTokenizer} filtered by a {@link
        /// StandardFilter}, a {@link LowerCaseFilter} and a {@link StopFilter}. 
        /// </summary>
        public override TokenStream TokenStream(System.String fieldName, System.IO.TextReader reader)
        {
            TokenStream result = new ChineseTokenizer(reader);
            result = new StandardFilter(result);
            //result = new LowerCaseFilter(result);
            result = new StopFilter(result, CHINESE_ENGLISH_STOP_WORDS);
            return result;
        }
    }
}
